﻿/****************************** Module Header ******************************\
 * Module Name:  SpiderHelper.cs
 * Module Version:  1.0.0 (2013-8-22)
 * Copyright (c) AnyEB.Com (14507247@qq.com)
 * 
 * This class is used to regularexpressions validator.  It supplies 
 * most validator ruler define.
 * 
 * This source is subject to the GPL License.
 * See http://www.gnu.org/licenses/gpl.html.
 * All other rights reserved.
 * 
 * THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, 
 * EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED 
 * WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
\***************************************************************************/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Data;
using System.Text;
using System.Net;
using System.IO;

namespace UCF.Core
{
    /// <summary>
    /// 网页抓取助手类
    /// </summary>
    /// <remarks></remarks>
    /// <example>
    /// CookieContainer cc = new CookieContainer(); //this is for keep the Session and Cookie
    /// Hashtable param = new Hashtable(); //this is for keep post data.
    /// 
    /// string urlLogin = "http://demo.server//login.asp";
    /// //do find the elementId that needed. check the source of login page can get this information
    /// param.Add("User", "xxx");
    /// param.Add("Password", "xxxx");
    /// string result =GrabHelper.PostAndGetHTML(urlLogin, cc, param);
    /// //check result, whether login success
    /// 
    /// //if login success, goto the target url, and input some value.
    /// string url2 = " http://demo.server/query.asp?id=1"; //need change. special logic
    /// param.Clear();
    /// //param.Add("SearchAreaId","JobId")
    /// result = GrabHelper.PostAndGetHTML(url2, cc, new Hashtable());
    /// //ConvertToDT the html or do something others
    /// </example>
    public class SpiderHelper
    {
        /// <summary>
        /// 错误消息
        /// </summary>
        public static string Error;

        /// <summary>
        /// 获取网页内容
        /// <para>Get("http://117.25.177.101:8899/xcall/index.php/XCall/ApiCallRecord/seat_screen_popup/exten/333/uid/358.html");</para>
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        /// <remarks>2015-1-26 by sutroon</remarks>
        public static string Get(string url)
        {
            try
            {
                WebClient wc = new WebClient();
                wc.Credentials = CredentialCache.DefaultCredentials;
                Byte[] pageData = wc.DownloadData(url);
                // return = Encoding.Default.GetString(pageData);  //gb2312
                return Encoding.UTF8.GetString(pageData); //utf-8
            }
            catch (WebException webEx)
            {
                Error = webEx.Message.ToString();
            }
            return null;
        }
        public static void GetAsync(string url, Action<string> handler)
        {
            try
            {
                WebClient wc = new WebClient();
                wc.Credentials = CredentialCache.DefaultCredentials;
                wc.DownloadDataAsync(new Uri(url));
                wc.DownloadDataCompleted += (object sender, DownloadDataCompletedEventArgs e) =>
                {
                    Byte[] pageData = e.Result;
                    handler(Encoding.UTF8.GetString(pageData));
                };
            }
            catch (WebException webEx)
            {
                Error = webEx.Message.ToString();
            }
        }

        /// <summary>
        /// 用Post获取网页内容
        /// </summary>
        /// <param name="url">地址, 如: http://localhost/register.php </param>
        /// <param name="post">参数, 如：arg1=a&arg2=b</param>
        /// <returns></returns>
        /// <remarks>2015-2-12 by [http://blog.csdn.net/xizhibei/article/details/6972116]</remarks>
        public static string Post(string url, string post)
        {
            try
            {
                byte[] postData = Encoding.UTF8.GetBytes(post);//编码，尤其是汉字，事先要看下抓取网页的编码方式
                WebClient webClient = new WebClient();
                webClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");//采取POST方式必须加的header，如果改为GET方式的话就去掉这句话即可  
                byte[] responseData = webClient.UploadData(url, "POST", postData);//得到返回字符流  
                return Encoding.UTF8.GetString(responseData);//解码  
            }
            catch (WebException webEx)
            {
                Error = webEx.Message.ToString();
            }
            return null;
        }


        /// <summary>
        /// 获取指定网址的源码
        /// </summary>
        /// <param name="targetURL"></param>
        /// <param name="cc">this is for keeping cookies and sessions</param>
        /// <param name="param">this is the data need post inside form</param>
        /// <param name="charset">this is for the charset</param>
        /// <returns>html page</returns>
        public static string PostAndGetHTML(string targetURL, CookieContainer cc, Dictionary<string, string> param, string charset = "utf-8")
        {
            if (charset.Length == 0) charset = "utf-8";
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(targetURL);
            request.Method = "POST";    //post
            request.ContentType = "application/x-www-form-urlencoded";
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.1124)";

            //prepare the submit data
            if (param != null)
            {
                string formData = "";
                foreach (string k in param.Keys)
                {
                    formData += k + "=" + param[k] + "&";
                }
                if (formData.Length > 0) formData = formData.Substring(0, formData.Length - 1); //remove last '&'

                ASCIIEncoding encoding = new ASCIIEncoding();
                byte[] data = encoding.GetBytes(formData);

                request.ContentLength = data.Length;

                Stream newStream = request.GetRequestStream();
                newStream.Write(data, 0, data.Length);
                newStream.Close();
            }

            if (cc != null) request.CookieContainer = cc;
            try
            {
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                if (cc != null) cc.Add(response.Cookies);
                Stream stream = response.GetResponseStream();
                return new StreamReader(stream, Encoding.GetEncoding(charset)).ReadToEnd();
            }
            catch (Exception)
            {
                return "error";
            }
        }

        // 测试Url状态是否正常
        public static string TestUrl(string url)
        {
            try
            {
                HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
                HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
                if (resp.StatusCode == HttpStatusCode.OK)
                {
                    resp.Close();
                    return "success";
                }
                return resp.StatusDescription;
            }
            catch (WebException ex)
            {
                return ex.Message;
            }
        }

        public static DataTable ConvertToDT(DataTable dt, string tableHTML)
        {

            int lastTD = tableHTML.ToLower().LastIndexOf("</td>");
            int firstRow = tableHTML.ToLower().IndexOf("<tr") + 3;//after ""<tr
            int index = tableHTML.ToLower().IndexOf("<tr", firstRow) + 3;//after ""<tr
            while (index < lastTD)
            {
                DataRow dr = dt.NewRow();
                for (int i = 0; i < dt.Columns.Count; i++)
                {
                    string value = "";
                    int startTD = tableHTML.ToLower().IndexOf("<td", index) + 3;//after "<td"
                    int endTD = tableHTML.ToLower().IndexOf("</td>", startTD);
                    if (endTD < 0)
                        break;
                    string tdStr = tableHTML.Substring(startTD, endTD - startTD);

                    //remove <> and others
                    tdStr = tdStr.Replace("&nbsp;", "").Replace("\t", "").Replace("\r", "");
                    string[] v = tdStr.Split('<', '>');
                    for (int j = 0; j < v.Length; j++)
                    {
                        j++;
                        if (v[j].Trim() != "")
                        {
                            value = v[j].Trim();
                            break;
                        }
                    }
                    //
                    dr[i] = value;
                    index = endTD;
                }
                dt.Rows.Add(dr);

            }
            return dt;
        }
    }
}
